//-------------------------------------------------------------------------//
//                                                                         //
//  This benchmark is an OpenMP C version of the NPB SP code. This OpenMP  //
//  C version is developed by the Center for Manycore Programming at Seoul //
//  National University and derived from the OpenMP Fortran versions in    //
//  "NPB3.3-OMP" developed by NAS.                                         //
//                                                                         //
//  Permission to use, copy, distribute and modify this software for any   //
//  purpose with or without fee is hereby granted. This software is        //
//  provided "as is" without express or implied warranty.                  //
//                                                                         //
//  Information on NPB 3.3, including the technical report, the original   //
//  specifications, source code, results and information on how to submit  //
//  new results, is available at:                                          //
//                                                                         //
//           http://www.nas.nasa.gov/Software/NPB/                         //
//                                                                         //
//  Send comments or suggestions for this OpenMP C version to              //
//  cmp@aces.snu.ac.kr                                                     //
//                                                                         //
//          Center for Manycore Programming                                //
//          School of Computer Science and Engineering                     //
//          Seoul National University                                      //
//          Seoul 151-744, Korea                                           //
//                                                                         //
//          E-mail:  cmp@aces.snu.ac.kr                                    //
//                                                                         //
//-------------------------------------------------------------------------//

//-------------------------------------------------------------------------//
// Authors: Sangmin Seo, Jungwon Kim, Jun Lee, Jeongho Nah, Gangwon Jo,    //
//          and Jaejin Lee                                                 //
//-------------------------------------------------------------------------//

#include "header.h"
#include "mpinpb.h"
//---------------------------------------------------------------------
// addition of update to the vector u
//---------------------------------------------------------------------
void add()
{
  int i, j, k, m;
  int scount1[total_nodes], rcount1[total_nodes], displ1[total_nodes];
  int kstart, kend,kgap,krem;
  MPI_Status status;
  MPI_Request sendrequest, recvrequest;

  if (timeron) timer_start(t_add);
  //#pragma omp parallel for default(shared) private(i,j,k,m)
  kgap = (grid_points[2])/total_nodes;
  krem = (grid_points[2])%total_nodes;
  if(krem == 0){
	kstart=node*kgap;
	kend= kstart+kgap;}
  else {
	  if(node+1 <= krem) 
	  {
		kstart = node*(kgap+1); //1 , 4
		kend= kstart+kgap+1; // 4, 7,
	  }
	  else
	  {
		kstart= krem*(kgap+1)+(node - krem)*kgap; //	7 , 9
		kend = kstart +kgap; // 9, 11
	  } 
	}
  
  if(node == root) kstart = kstart+1;
  if(node == total_nodes-1) kend= kend -1;

  for (k = kstart; k <  kend; k++) { 
    for (j = 1; j <= ny2; j++) {
      for (i = 1; i <= nx2; i++) {
        for (m = 0; m < 5; m++) {
          u[k][j][i][m] = u[k][j][i][m] + rhs[k][j][i][m];
        }
      }
    }
  }
  if(node == root) kstart = kstart-1;
  if(node == total_nodes-1) kend= kend +1;

  if (timeron) timer_stop(t_add);	

  //u array k+2
if(node > 0) MPI_Isend(&u[kstart][0][0], 2*(JMAXP+1)*(IMAXP+1)*5,MPI_DOUBLE, node-1, 1, MPI_COMM_WORLD, &sendrequest);
if(node < total_nodes -1) MPI_Irecv(&u[kend][0][0], 2*(JMAXP+1)*(IMAXP+1)*5,MPI_DOUBLE, node+1,1, MPI_COMM_WORLD,&recvrequest);

//u array k-2
 if(node < total_nodes -1)  MPI_Isend(&u[kend-2][0][0], 2*(JMAXP+1)*(IMAXP+1)*5,MPI_DOUBLE, node+1,2, MPI_COMM_WORLD, &sendrequest);
 if (node > 0)  MPI_Irecv(&u[kstart-2][0][0], 2*(JMAXP+1)*(IMAXP+1)*5,MPI_DOUBLE,node-1,2, MPI_COMM_WORLD,&recvrequest);

}

